﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;

using LemmaSharp;

namespace SearchingWebServices
{
    public static class Tokenizer
    {
        public static readonly LemmatizerPrebuiltCompact Lemmatizer = new LemmatizerPrebuiltCompact(LanguagePrebuilt.English);

        // List of stop words from http://norm.al/2009/04/14/list-of-english-stop-words/
        private const string StopWords = "a,able,about,across,after,all,almost,also,am,among,an,and,any,are,as,at,be,because,been,but,by,can,cannot,could,dear,did,do,does,either,else,ever,every,for,from,get,got,had,has,have,he,her,hers,him,his,how,however,i,if,in,into,is,it,its,just,least,let,like,likely,may,me,might,most,must,my,neither,no,nor,not,of,off,often,on,only,or,other,our,own,rather,said,say,says,she,should,since,so,some,than,that,the,their,them,then,there,these,they,this,tis,to,too,twas,us,wants,was,we,were,what,when,where,which,while,who,whom,why,will,with,would,yet,you,your";
        private static readonly HashSet<string> StopWordsSet = new HashSet<string>(StopWords.Split(','), StringComparer.InvariantCultureIgnoreCase);
        private static readonly char[] SplitChars = { ' ', '_', ',', '!', '@', '#', '$', '%', '^', '&', '*', '(', ')', '=', '+', '{', '}', '[', ']', '|', '\\', '/', ':', ';', '"', '<', '>', '.', '?', '`', '~', '-', '\r', '\n', '\t' };

        public static List<string> Tokenize(string input)
        {
            // Return empty list if input is empty
            if (string.IsNullOrWhiteSpace(input))
            {
                return new List<string>();
            }

            // Remove numbers
            input = Regex.Replace(input, "[0-9]", string.Empty);

            // Initialize wordset with collection of parts created by splitting on spaces, underscores, etc.
            // This also serves to remove special characters from the text
            var wordSet = new HashSet<string>(input.Split(SplitChars, StringSplitOptions.RemoveEmptyEntries), StringComparer.InvariantCultureIgnoreCase);

            // Split words at case changes. This will repeat until no more splits occur.
            bool repeat;
            var part = new StringBuilder();

            do
            {
                repeat = false;
                
                foreach (var word in wordSet.ToList())
                {
                    part.Clear();
                    var previousCapital = false;

                    for (var i = 0; i < word.Length; i++)
                    {
                        var chr = word[i];
                        var next = i + 1 < word.Length
                                       ? word[i + 1]
                                       : ' ';

                        if (part.Length == 0)
                        {
                            part.Append(chr);
                            previousCapital = char.IsUpper(chr);
                            continue;
                        }

                        // If next character is upper and the previous wasn't,
                        // then we need to split.  Tacking together multiple capitals
                        // for situations such as acronyms (e.g. SQL, WSDL), which can
                        // be acceptable terms in variable names. The IsUpper check on
                        // the next character allows a split for something like SQLStuff into SQL and Stuff
                        if (char.IsUpper(chr) && (!previousCapital || (next != ' ' && !char.IsUpper(next))))
                        {
                            var partString = part.ToString();
                            if (!wordSet.Contains(partString))
                            {
                                wordSet.Add(partString);
                                repeat = true;
                            }

                            // Remove original word, since we are splitting it up
                            if (wordSet.Contains(word))
                            {
                                wordSet.Remove(word);
                            }

                            part.Clear();
                        }

                        part.Append(chr);
                        previousCapital = char.IsUpper(chr);
                    }

                    if (part.Length > 0)
                    {
                        var partString = part.ToString();
                        if (!wordSet.Contains(partString))
                        {
                            // Remove original word, since we are splitting it up
                            if (wordSet.Contains(word))
                            {
                                wordSet.Remove(word);
                            }

                            wordSet.Add(partString);
                            repeat = true;
                        }
                    }
                }
            } while (repeat);

            // Remove stop words (the, is, and, or, etc...) and return
            return wordSet.Where(word => !StopWordsSet.Contains(word)).ToList();
        }
    }
}
